#if ( !defined __CRAWLER_HPP__ )
#define __CRAWLER_HPP__

#define SQL_CREATE_WEBPAGES_TABLE "CREATE TABLE t_webpages(\
								id INTEGER PRIMARY KEY, \
								url VARCHAR(200) UNIQUE NOT NULL, \
								page_header TEXT, \
								page_content TEXT, \
								create_time DATE, \
								update_time DATE );"

#define SQL_UPDATE_WEBPAGES "UPDATE t_webpages \
						SET \
							page_header=?, \
							page_content=?, \
							update_time=DATETIME('NOW') \
						WHERE url=?;"

#define SQL_INSERT_WEBPAGES "INSERT INTO t_webpages (\
							url, \
							page_header, \
							page_content, \
							create_time, \
							update_time \
						)VALUES( \
							?, \
							?, \
							?, \
							DATETIME('NOW'), \
							DATETIME('NOW') \
						);"

#define SQL_SELECT_WEBPAGES_WITH_URL "SELECT url,page_header,page_content,create_time,update_time FROM t_webpages WHERE url=?;"
#define SQL_SELECT_WEBPAGES_COUNT_WITH_URL "SELECT count(*) FROM t_webpages WHERE url=?;"

#include <string>
#include <vector>

#include "url_seed.hpp"
#include "webpage_hunter.hpp"
#include "sqlite3/sqlite3_wrapper.hpp"

using namespace std;
using namespace sqlite3_wrapper;

class Crawler
{
public:
	Crawler ( const string& protocol, const string& host, const vector< string >& paths, const HttpHeaderPairs& headers, const vector< string >& proxies, const vector < string >& expressions );
	~Crawler ();

public:
	void operator()();

protected:
	void use_proxy ( string& server, string& port );
	void save_webpage ( const string& url, const string& page_header, const string& page_content );
	
	void create_database ();
	void update_webpage ( const string& url, const string& page_header, const string& page_content );
	void read_webpage ( const string& url, stringstream& header_stream, stringstream& content_stream, const string& server, const string& port, const string& host, const string& path, const HttpHeaderPairs& headers );
private:
	UrlSeed* _seed;
	HttpHeaderPairs _headers;
	vector < string > _proxies;
	sqlite3_connection* _db_connection;
};

#endif // __CRAWLER_HPP__